Switch to multi-step chunk addition to pave the way for streaming

compression

Signed-off-by: Jonathan Dieter <jdieter@gmail.com>

diff --git a/src/lib/comp/comp.c b/src/lib/comp/comp.c
index 831555165dff3168b3b79a190a7b52b0a042f866..11c75062711518fe1232b07ef1f3f63022ef8464 100644 (file)
--- a/src/lib/comp/comp.c
+++ b/src/lib/comp/comp.c
@@ -38,7 +38,7 @@
 
 #define BLK_SIZE 32768
 
-static char unknown[] = "Unknown(\0\0\0\0\0";
+static char unknown[] = "Unknown(\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
 
 const static char *COMP_NAME[] = {
     "no",
@@ -71,7 +71,8 @@ int zck_comp_init(zckCtx *zck) {
             free(dst);
             return False;
         }
-        zck_index_add_chunk(zck, dst, dst_size, zck->comp.dict_size);
+        zck_index_add_to_chunk(zck, dst, dst_size, zck->comp.dict_size);
+        zck_index_finish_chunk(zck);
         free(dst);
     }
     zck->comp.dict = NULL;
@@ -100,7 +101,8 @@ int zck_compress(zckCtx *zck, const char *src, const size_t src_size) {
         free(dst);
         return False;
     }
-    zck_index_add_chunk(zck, dst, dst_size, src_size);
+    zck_index_add_to_chunk(zck, dst, dst_size, src_size);
+    zck_index_finish_chunk(zck);
     free(dst);
     return True;
 }
@@ -184,7 +186,7 @@ int zck_set_comp_parameter(zckCtx *zck, int option, void *value) {
 
 const char *zck_comp_name_from_type(int comp_type) {
     if(comp_type > 1) {
-        snprintf(unknown+8, 4, "%i)", comp_type);
+        snprintf(unknown+8, 21, "%i)", comp_type);
         return unknown;
     }
     return COMP_NAME[comp_type];

diff --git a/src/lib/index/index_common.c b/src/lib/index/index_common.c
index 9435ce326304cdfc86840ffabbd585edf37256ca..04917d366e2ece53c749a346285dd15897397287 100644 (file)
--- a/src/lib/index/index_common.c
+++ b/src/lib/index/index_common.c
@@ -31,6 +31,17 @@
 
 #include "zck_private.h"
 
+void zck_index_free_item(zckIndexItem **item) {
+    if(*item == NULL)
+        return;
+
+    if((*item)->digest)
+        free((*item)->digest);
+    free(*item);
+    *item = NULL;
+    return;
+}
+
 void zck_index_clean(zckIndex *index) {
     if(index == NULL)
         return;
@@ -40,9 +51,7 @@ void zck_index_clean(zckIndex *index) {
         zckIndexItem *tmp=index->first;
         while(tmp != NULL) {
             next = tmp->next;
-            if(tmp->digest)
-                free(tmp->digest);
-            free(tmp);
+            zck_index_free_item(&tmp);
             tmp = next;
         }
     }

diff --git a/src/lib/index/index_create.c b/src/lib/index/index_create.c
index b50e9d592c0c39f9ab2fdd2a4cd67db281b67b5e..9902b380c651f98570367d9d657feb75219dce56 100644 (file)
--- a/src/lib/index/index_create.c
+++ b/src/lib/index/index_create.c
@@ -31,7 +31,14 @@
 
 #include "zck_private.h"
 
+#define VALIDATE(f)     if(!f) { \
+                            zck_log(ZCK_LOG_ERROR, "zckCtx not initialized\n"); \
+                            return False; \
+                        }
+
 int zck_index_finalize(zckCtx *zck) {
+    VALIDATE(zck);
+
     zckHash index_hash;
     char *index;
     size_t index_malloc = 0;
@@ -121,6 +128,36 @@ int zck_index_finalize(zckCtx *zck) {
     return True;
 }
 
+static int finish_chunk(zckIndex *index, zckIndexItem *item, char *digest,
+                        int finished) {
+    VALIDATE(index);
+    VALIDATE(item);
+
+    item->digest = zmalloc(index->digest_size);
+    if(item->digest == NULL) {
+        zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n",
+                index->digest_size);
+        return False;
+    }
+    if(digest) {
+        memcpy(item->digest, digest, index->digest_size);
+        item->digest_size = index->digest_size;
+    }
+    item->start = index->length;
+    item->finished = finished;
+    if(index->first == NULL) {
+        index->first = item;
+    } else {
+        zckIndexItem *tmp = index->first;
+        while(tmp->next)
+            tmp = tmp->next;
+        tmp->next = item;
+    }
+    index->count += 1;
+    index->length += item->comp_length;
+    return True;
+}
+
 int zck_index_new_chunk(zckIndex *index, char *digest, int digest_size,
                         size_t comp_size, size_t orig_size, int finished) {
     if(index == NULL) {
@@ -137,64 +174,68 @@ int zck_index_new_chunk(zckIndex *index, char *digest, int digest_size,
                 sizeof(zckIndexItem));
         return False;
     }
-    idx->digest = zmalloc(digest_size);
-    if(idx->digest == NULL) {
-        zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", digest_size);
-        return False;
-    }
-    if(digest)
-        memcpy(idx->digest, digest, digest_size);
-    idx->digest_size = digest_size;
-    idx->start = index->length;
+    index->digest_size = digest_size;
     idx->comp_length = comp_size;
     idx->length = orig_size;
-    idx->finished = finished;
-    if(index->first == NULL) {
-        index->first = idx;
-    } else {
-        zckIndexItem *tmp=index->first;
-        while(tmp->next)
-            tmp = tmp->next;
-        tmp->next = idx;
+    return finish_chunk(index, idx, digest, finished);
+}
+
+int zck_index_create_chunk(zckCtx *zck) {
+    VALIDATE(zck);
+
+    zck_clear_work_index(zck);
+    zck->work_index_item = zmalloc(sizeof(zckIndexItem));
+    if(zck->work_index_item == NULL) {
+        zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n",
+                sizeof(zckIndexItem));
+        return False;
     }
-    index->count += 1;
-    index->length += comp_size;
+    if(!zck_hash_init(&(zck->work_index_hash), &(zck->chunk_hash_type)))
+        return False;
     return True;
 }
 
-int zck_index_add_chunk(zckCtx *zck, char *data, size_t comp_size,
-                        size_t orig_size) {
-    zckHash hash;
+int zck_index_add_to_chunk(zckCtx *zck, char *data, size_t comp_size,
+                           size_t orig_size) {
+    VALIDATE(zck);
 
-    if(zck == NULL) {
-        zck_log(ZCK_LOG_ERROR, "Invalid zck context");
+    if(comp_size == 0)
+        return True;
+
+    if(zck->work_index_item == NULL && !zck_index_create_chunk(zck))
         return False;
-    }
 
-    if(comp_size == 0) {
-        if(!zck_index_new_chunk(&(zck->index), NULL, zck->index.digest_size,
-                                comp_size, orig_size, True))
-            return False;
-    } else {
-        if(!zck_hash_update(&(zck->full_hash), data, comp_size))
-            return False;
-        if(!zck_hash_init(&hash, &(zck->chunk_hash_type)))
-            return False;
-        if(!zck_hash_update(&hash, data, comp_size))
-            return False;
-
-        char *digest = zck_hash_finalize(&hash);
-        if(digest == NULL) {
-            zck_log(ZCK_LOG_ERROR,
-                    "Unable to calculate %s checksum for new chunk\n",
-                    zck_hash_name_from_type(zck->index.hash_type));
-            return False;
-        }
-        if(!zck_index_new_chunk(&(zck->index), digest, zck->index.digest_size,
-                                comp_size, orig_size, True))
-            return False;
-        free(digest);
+    if(!zck_hash_update(&(zck->full_hash), data, comp_size))
+        return False;
+    if(!zck_hash_update(&(zck->work_index_hash), data, comp_size))
+        return False;
+
+    zck->work_index_item->comp_length += comp_size;
+    zck->work_index_item->length += orig_size;
+    return True;
+}
+
+int zck_index_finish_chunk(zckCtx *zck) {
+    VALIDATE(zck);
+
+    if(zck->work_index_item == NULL && !zck_index_create_chunk(zck))
+        return False;
+
+    /* Finalize chunk checksum */
+    char *digest = zck_hash_finalize(&(zck->work_index_hash));
+    if(digest == NULL) {
+        zck_log(ZCK_LOG_ERROR,
+                "Unable to calculate %s checksum for new chunk\n",
+                zck_hash_name_from_type(zck->index.hash_type));
+        return False;
     }
+
+    if(!finish_chunk(&(zck->index), zck->work_index_item, digest, True))
+        return False;
+
+    free(digest);
+    zck->work_index_item = NULL;
+    zck_hash_close(&(zck->work_index_hash));
     return True;
 }
 

diff --git a/src/lib/zck.c b/src/lib/zck.c
index 99bd04504887d8550b35a61ddc09380c0f92d089..d236b1b53f9250154d05a8c127150d1ac01cf339 100644 (file)
--- a/src/lib/zck.c
+++ b/src/lib/zck.c
@@ -62,6 +62,15 @@ int zck_write_file(zckCtx *zck) {
     return True;
 }
 
+void zck_clear_work_index(zckCtx *zck) {
+    if(zck == NULL)
+        return;
+
+    zck_hash_close(&(zck->work_index_hash));
+    if(zck->work_index_item)
+        zck_index_free_item(&(zck->work_index_item));
+}
+
 void zck_clear(zckCtx *zck) {
     if(zck == NULL)
         return;
@@ -69,6 +78,7 @@ void zck_clear(zckCtx *zck) {
     zck_comp_close(zck);
     zck_hash_close(&(zck->full_hash));
     zck_hash_close(&(zck->check_full_hash));
+    zck_clear_work_index(zck);
     if(zck->full_hash_digest) {
         free(zck->full_hash_digest);
         zck->full_hash_digest = NULL;

diff --git a/src/lib/zck_private.h b/src/lib/zck_private.h
index 75465f5359cc7b98d64f65cbabf4275b6d933df5..80f8c899351b461159d8cf50b5fa2a37e8baaa15 100644 (file)
--- a/src/lib/zck_private.h
+++ b/src/lib/zck_private.h
@@ -88,6 +88,8 @@ typedef struct zckCtx {
     char *index_string;
     size_t index_size;
     zckIndex index;
+    zckIndexItem *work_index_item;
+    zckHash work_index_hash;
     char *index_digest;
     zckHash full_hash;
     zckHash check_full_hash;
@@ -99,6 +101,7 @@ typedef struct zckCtx {
 const char *zck_hash_name_from_type(int hash_type);
 int zck_get_tmp_fd();
 int zck_validate_file(zckCtx *zck);
+void zck_clear_work_index(zckCtx *zck);
 
 /* hash/hash.h */
 int zck_hash_setup(zckHashType *ht, int h);
@@ -113,10 +116,12 @@ int zck_index_read(zckCtx *zck, char *data, size_t size);
 int zck_index_finalize(zckCtx *zck);
 int zck_index_new_chunk(zckIndex *index, char *digest, int digest_size,
                         size_t comp_size, size_t orig_size, int finished);
-int zck_index_add_chunk(zckCtx *zck, char *data, size_t comp_size,
+int zck_index_add_to_chunk(zckCtx *zck, char *data, size_t comp_size,
                         size_t orig_size);
+int zck_index_finish_chunk(zckCtx *zck);
 void zck_index_clean(zckIndex *index);
 void zck_index_free(zckCtx *zck);
+void zck_index_free_item(zckIndexItem **item);
 int zck_write_index(zckCtx *zck);
 
 /* io.c */

diff --git a/src/zck_dl.c b/src/zck_dl.c
index bb4b24e7e77a47459f4c312ad066e69f6fc55432..cfc6a63f6c97be44dff8c2f7fd28d782fb3aa225 100644 (file)
--- a/src/zck_dl.c
+++ b/src/zck_dl.c
@@ -95,12 +95,13 @@ int main (int argc, char *argv[]) {
 
 
     printf("Downloaded %lu bytes\n", zck_dl_get_bytes_downloaded(dl));
+    int exit_val = 0;
     switch(zck_hash_check_data(dl->zck, dl->dst_fd)) {
         case -1:
-            exit(1);
+            exit_val = 1;
             break;
         case 0:
-            exit(1);
+            exit_val = 1;
             break;
         default:
             break;
@@ -109,5 +110,5 @@ int main (int argc, char *argv[]) {
     zck_free(&zck_tgt);
     zck_free(&zck_src);
     zck_dl_global_cleanup();
-    exit(0);
+    exit(exit_val);
 }